import csv
from operator import length_hint

import numpy as np
import pandas as pd
from numpy import dtype


# 预处理数据
def preprocessor_train_data(csvfile_path_list):
    raw_data = pd.DataFrame()
    for path in csvfile_path_list:
        data = pd.read_csv(path)
        raw_data = pd.concat([raw_data, data], axis=1)
    if "income" in raw_data.columns:
        raw_data = raw_data.drop(columns=["income"])
    # 读取非数值列
    list_object_columns = [col for col in raw_data.columns if raw_data[col].dtypes == "object"]
    object_data = raw_data[list_object_columns]
    # 读取数值列
    list_number_columns = [col for col in raw_data.columns if raw_data[col].dtypes != "object"]
    number_data = raw_data[list_number_columns]

    # 对非数值列进行独热编码
    object_data = pd.get_dummies(object_data)
    raw_data = pd.concat([object_data, number_data], axis=1)

    raw_data = raw_data.astype("int64")

    # 归一化
    x_train = (raw_data - raw_data.mean()) / raw_data.std()

    y_train = pd.read_csv('data/train/Y_train.csv')
    y_train = np.array(y_train).flatten()
    y_train = pd.DataFrame(y_train,columns=['income']).astype('int64')

    return x_train, y_train


def normalize_column(x, train=True, specified_column=None, x_mean=None, x_std=None):
    # 将训练数据集中归一化到0周围，并且符合正态分布
    if train:
        if specified_column is None:
            specified_column = np.arange(x.shape[1])
        length = len(specified_column)
        x_mean = np.reshape(np.mean(x[:, specified_column], 0), (1, length))
        x_std = np.reshape(np.std(x[:, specified_column], 0), (1, length))
    x[:, specified_column] = np.divide(np.subtract(x[:, specified_column], x_mean), x_std)

    return x

def preprocessor_test_data(csvfile_path_list):
    raw_data = pd.DataFrame()
    for path in csvfile_path_list:
        data = pd.read_csv(path)
        raw_data = pd.concat([raw_data, data], axis=1)
    if "income" in raw_data.columns:
        raw_data = raw_data.drop(columns=["income"])
    # 读取非数值列
    list_object_columns = [col for col in raw_data.columns if raw_data[col].dtypes == "object"]
    object_data = raw_data[list_object_columns]
    # 读取数值列
    list_number_columns = [col for col in raw_data.columns if raw_data[col].dtypes != "object"]
    number_data = raw_data[list_number_columns]

    # 对非数值列进行独热编码
    object_data = pd.get_dummies(object_data)
    raw_data = pd.concat([object_data, number_data], axis=1)

    raw_data = raw_data.astype("int64")

    # 归一化
    x_test = (raw_data - raw_data.mean()) / raw_data.std()

    return x_test